package org.nlp2rdf.navigator.benchmark.cli;

import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import com.hp.hpl.jena.rdf.model.*;
import com.hp.hpl.jena.vocabulary.RDF;
import org.aksw.commons.util.SerializationUtils;
import org.aksw.commons.util.experiments.ExperimentStatisticsCollector;
import org.aksw.commons.util.random.RandomUtils;
import org.aksw.commons.util.strings.StringUtils;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.dllearner.core.EvaluatedDescription;
import org.nlp2rdf.navigator.benchmark.*;
import org.nlp2rdf.navigator.benchmark.pools.IPoolFactory;
import org.nlp2rdf.navigator.benchmark.pools.RandomPoolFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.util.*;


class Scovo {
    public static final String namespace = "http://purl.org/NET/scovo#";
    public static final Resource Item = ResourceFactory.createResource(namespace + "Item");
    public static final Resource Dataset = ResourceFactory.createResource(namespace + "Dataset");
    public static final Resource Dimension = ResourceFactory.createResource(namespace + "Dimension");

    public static final Property hasDimension = ResourceFactory.createProperty(namespace + "dimension");
    public static final Property hasDataset = ResourceFactory.createProperty(namespace + "dataset");
    public static final Property isDatasetOf = ResourceFactory.createProperty(namespace + "datasetOf");
    public static final Property hasMin = ResourceFactory.createProperty(namespace + "min");
    public static final Property hasMax = ResourceFactory.createProperty(namespace + "max");

}

class Vocab {
    public static final String namespace = "http://nke.org/";
    public static final Resource Precision = ResourceFactory.createResource(namespace + "Precision");
    public static final Resource Recall = ResourceFactory.createResource(namespace + "Recall");
    public static final Resource FMeasure = ResourceFactory.createResource(namespace + "FMeasure");

    public static final Resource LearningAccuracy = ResourceFactory.createResource(namespace + "LearningAccuracy");
    public static final Resource LearnedConcept = ResourceFactory.createResource(namespace + "LearnedConcept");
    public static final Resource InstanceCount = ResourceFactory.createResource(namespace + "InstanceCount");
    public static final Resource RetrievedInstanceCount = ResourceFactory.createResource(namespace + "RetrievedInstanceCount");

    //public static final Resource TrainPositive = ResourceFactory.createResource(namespace + "TrainPositive");
    //public static final Resource TrainNegative = ResourceFactory.createResource(namespace + "TrainNegative");

    public static final Resource TestPositive = ResourceFactory.createResource(namespace + "TestPositive");
    public static final Resource TestNegative = ResourceFactory.createResource(namespace + "TestNegative");

    public static final Resource NumTrainPositive = ResourceFactory.createResource(namespace + "NumTrainPositive");
    public static final Resource NumTrainNegative = ResourceFactory.createResource(namespace + "NumTrainNegative");

    public static final Resource NumTestPositive = ResourceFactory.createResource(namespace + "NumTestPositive");
    public static final Resource NumTestNegative = ResourceFactory.createResource(namespace + "NumTestNegative");


    public static final Resource Phase = ResourceFactory.createResource(namespace + "Phase");

    public static final Resource StatsAll = ResourceFactory.createResource(namespace + "StatsAll");
    public static final Resource StatsAllTyped = ResourceFactory.createResource(namespace + "StatsAllTyped");
    public static final Resource StatsTest = ResourceFactory.createResource(namespace + "StatsTest");
    public static final Resource StatsTestTyped = ResourceFactory.createResource(namespace + "StatsTestTyped");

    public static final Property hasStats = ResourceFactory.createProperty(namespace + "hasStats");
    public static final Property hasError = ResourceFactory.createProperty(namespace + "hasError");

    public static final Property hasCategoryId = ResourceFactory.createProperty(namespace + "hasCategoryId");
    public static final Property hasFoldId = ResourceFactory.createProperty(namespace + "hasFoldId");
    public static final Property hasPhaseId = ResourceFactory.createProperty(namespace + "hasPhaseId");


    public static final Property hasTrainPositives = ResourceFactory.createProperty(namespace + "hasTrainPositives");
    public static final Property hasTrainNegatives = ResourceFactory.createProperty(namespace + "hasTrainNegatives");

    public static final Property hasFalsePositives = ResourceFactory.createProperty(namespace + "hasFalsePositives");
    public static final Property hasFalseNegatives = ResourceFactory.createProperty(namespace + "hasFalseNegatives");

    public static final Property numTruePositives = ResourceFactory.createProperty(namespace + "numTruePositives");
    public static final Property numTrueNegatives = ResourceFactory.createProperty(namespace + "numTrueNegatives");
    public static final Property numFalsePositives = ResourceFactory.createProperty(namespace + "numFalsePositives");
    public static final Property numFalseNegatives = ResourceFactory.createProperty(namespace + "numFalseNegatives");
}

/**
 * Created by Claus Stadler
 * Date: Oct 23, 2010
 * Time: 6:47:08 PM
 */
public class RunBenchmark {
    private static final Logger logger = LoggerFactory.getLogger(ClausBenchmark.class);

    /*
    private static Options cliOptions = new Options()
            .addOption("f", "sampleFile", true, "Sample file")
            .addOption("c", "configFile", true, "Spring XML config")
            .addOption("k", "categoryName", true, "category name to search for")
            .addOption("e", "experimentName", true, "A name for this run")
            .addOption("r", "randomSeed", true, "random seed")
            .addOption("n", "numPhases", true, "number of phases")
            .addOption("p", "basePath", true, "basePath");
*/

    private static Options cliOptions = new Options()
            .addOption("s", "serviceConfig", true, "Spring XML config")
            .addOption("p", "poolFactoryName", true, "Spring XML config")
            .addOption("c", "category", true, "category uri")
            .addOption("e", "experimentName", true, "category uri");

    public static void main(String[] args)
            throws Exception {
        realMain(args);
        //oldRealMain(args);
        //testMain(args);
    }


    private static int pseudoBlankNode = 0;

    public static Resource newNode(Resource base) {
        return ResourceFactory.createResource(base.toString() + "/" + (++pseudoBlankNode));
    }


    /**
     * Creates a pseudo-blank node for the item and attaches it to the given subject.
     *
     * @param model
     * @param dimension
     * @param value
     */
    public static void writeValue(Model model, Resource dataset, Resource dimension, Object value) {
        Resource item = newNode(dataset);
        model.add(dataset, Scovo.isDatasetOf, item);
        //model.add(item, Scovo.hasDataset, dataset);
        model.add(item, RDF.type, Scovo.Item);
        model.add(item, Scovo.hasDimension, dimension);
        model.add(item, RDF.value, ResourceFactory.createTypedLiteral(value));
    }

    public static void write(Model model, Resource dataset, SampleStats<?> sampleStats) {
        writeValue(model, dataset, Vocab.Precision, sampleStats.getPrecision());
        writeValue(model, dataset, Vocab.Recall, sampleStats.getRecall());
        writeValue(model, dataset, Vocab.FMeasure, sampleStats.getFMeasure());
    }

    public static void write(Model model, Resource base, Resource type, SampleStats<?> sampleStats) {
        Resource node = newNode(base);
        model.add(base, Vocab.hasStats, node);

        model.add(node, Vocab.numTruePositives, ResourceFactory.createTypedLiteral(sampleStats.getTrue().getPositives().size()));
        //model.add(node, Vocab.numTrueNegatives, ResourceFactory.createTypedLiteral(sampleStats.getTrue().getNegatives().size()));
        model.add(node, Vocab.numFalsePositives, ResourceFactory.createTypedLiteral(sampleStats.getFalse().getPositives().size()));
        model.add(node, Vocab.numFalseNegatives, ResourceFactory.createTypedLiteral(sampleStats.getFalse().getNegatives().size()));

        // write the false negatives if there aren't too many of them
        {
            String value = sampleStats.getFalse().getNegatives().size() >= 50
                    ? ClausBenchmark.toMyString(sampleStats.getFalse().getNegatives(), 512)
                    : ClausBenchmark.toMyString(sampleStats.getFalse().getNegatives(), 0);

            model.add(node, Vocab.hasFalseNegatives, ResourceFactory.createPlainLiteral(value));
        }

        {
            String value = sampleStats.getFalse().getPositives().size() >= 50
                    ? ClausBenchmark.toMyString(sampleStats.getFalse().getPositives(), 512)
                    : ClausBenchmark.toMyString(sampleStats.getFalse().getPositives(), 0);

            model.add(node, Vocab.hasFalsePositives, ResourceFactory.createPlainLiteral(value));
        }


        model.add(node, RDF.type, type);
        write(model, node, sampleStats);
    }


    public static void writeCollection(Model model, Resource subject, Collection<?> collection) {

    }

    public static void writeSample(Model model, Resource subject, Sample<?> sample) {

    }

    public static void realMain(String[] args)
            throws Exception {
        CommandLine commandLine = CliUtils.getGnuCommandLine(cliOptions, args);

        //Date now = new Date();
        //String dateStr = now.toString().replace(" ", "_").replace(":", "_");

        String experimentName = commandLine.getOptionValue("e", "UnnamedExperiment");
        String serviceName = "service_" + commandLine.getOptionValue("s") + ".xml";
        String categoryName = commandLine.getOptionValue("c", "");
        String poolFactoryName = "poolFactories." + commandLine.getOptionValue("p", "");


        if (!categoryName.contains("://")) {
            if (!categoryName.contains("Category:")) {
                categoryName = "http://dbpedia.org/resource/Category:" + categoryName;
            } else {
                categoryName = "http://dbpedia.org/resource/" + categoryName;
            }
        }

        logger.info("Starting experiment: " + experimentName);
        logger.info("serviceName:     " + serviceName);
        logger.info("poolFactoryName: " + poolFactoryName);
        logger.info("categoryName:    " + categoryName);

        ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{
                serviceName, "navigator_dbpedia.xml", "benchmark_common.xml"});

        Resource category = ResourceFactory.createResource(categoryName);


        IPoolFactory poolFactory = (IPoolFactory) context.getBean(poolFactoryName);
        Random random = (Random) context.getBean("benchmark.random");

        Sample<Resource> pool = poolFactory.createPool(category);        

        BenchmarkContext con = (BenchmarkContext) context.getBean("benchmark.context");

        String outputBasePath = (String) context.getBean("benchmark.outputBasePath");

        Integer numPhases = (Integer) context.getBean("benchmark.numPhases");
        Integer numFolds = (Integer) context.getBean("benchmark.numFolds");
        IPhaseReconfigurator phaseReconfigurator = (IPhaseReconfigurator) context.getBean("benchmark.phaseReconfigurator");

        String namespace = "http://nke/" + experimentName + "/";



        // "Repair" pool: If the pool does not contain enough positives, add random ones in order
        // to keep the strategies a bit more comparable (rather than not performing at all, perform
        // as good or as bad as random)
        if(pool.getPositives().size() < numFolds || pool.getNegatives().size() < pool.getPositives().size()) {
            RandomPoolFactory randomPoolFactory = new RandomPoolFactory(con.getSparqlEndpoint(), con.getGraphNames(), random, 1000);

            Sample<Resource> randomPool = randomPoolFactory.createPool(category);

            //Sample<Resource> delta =
            int numExtraPositives = Math.max(0, numFolds - pool.getPositives().size());
            pool.getPositives().addAll(RandomUtils.randomSampleSet(randomPool.getPositives(), numExtraPositives, random));


            int numExtraNegatives = Math.max(0, pool.getPositives().size() - pool.getNegatives().size() + numFolds);


            pool.getNegatives().addAll(RandomUtils.randomSampleSet(randomPool.getNegatives(), numExtraNegatives, random));

            logger.info("Pool is too small, fixing with additional random values: +/-" + Joiner.on("/").join(numExtraPositives, numExtraNegatives));
        }
        

        try {
            runBenchmark(con, outputBasePath, namespace, numFolds, numPhases, category, pool, random, experimentName, phaseReconfigurator);
        } catch (Throwable t) {
            logger.error("Benchmarking of category " + category + " interrupted due to exception", t);
        }
    }

    public static void testMain(String[] args)
            throws Exception {
        ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{
                "service_hanne.xml", "navigator_dbpedia.xml", "benchmark_common.xml"});

        List<Integer> maxLearningTimeLimits = (List<Integer>) context.getBean("dbpedia.maxLearningTimeLimits");
        List<Integer> maxRetrievalLimis = (List<Integer>) context.getBean("dbpedia.maxRetrievalLimits");

        System.out.println(maxLearningTimeLimits);
        System.out.println(maxRetrievalLimis);


        BenchmarkContext con = new BenchmarkContext(context);
        Resource category = ResourceFactory.createResource("http://dbpedia.org/resource/Category:American_people_of_Danish_descent");

        Set<Resource> positives = MySparqlTasks.getTypedDBpediaCategoryMembers(con.getSparqlEndpoint(), con.getGraphNames(), category);




        //Sample<Resource> pool = PoolUtils.generateKeywordSearchPool(con.getSparqlEndpoint(), con.getGraphNames(), category.toString(), positives);
        Random random = new Random(0l);
        Sample<Resource> pool = PoolUtils.generateNavigationPool(con.getSparqlEndpoint(), con.getGraphNames(), category, positives, random);

        //simulateNavigation(context, "/tmp/Hanne Benchmark", 3, category, pool, random, "NoNameForExperiment");
    }


    public static void oldRealMain(String[] args)
            throws Exception {
        CommandLine commandLine = CliUtils.getGnuCommandLine(cliOptions, args);
        ApplicationContext context = CliUtils.getApplicationContext(commandLine, commandLine.getOptionValue("c"));

        BenchmarkContext con = new BenchmarkContext(context);

        Sample<Resource> pool = BenchmarkUtils.readSample(commandLine.getOptionValue("f"));

        int numPhases = Integer.parseInt(commandLine.getOptionValue("n", "3"));

        Resource category = ResourceFactory.createResource(commandLine.getOptionValue("k"));

        Random random = CliUtils.getRandom(commandLine.getOptionValue("r"));

        String experimentName = commandLine.getOptionValue("e");

        String basePath = commandLine.getOptionValue("p", "/tmp/BenchmarkResults");

        //Set<Resource> resources = MySparqlTasks.getTypedDBpediaCategoryMembers(con.getSparqlEndpoint(), con.getGraphNames(), category);

        //CliUtils.outputSet(resources);

        //simulateNavigation(con, basePath, numPhases, category, pool, random, experimentName);
    }

    /**
     * The output should probably be a table of this form
     * foldId   phaseId     instances learnedConcept
     * 1        1           12542            <>
     * ...
     * <p/>
     * <p/>
     * Or what would that look like with triples?
     * ns:ads5wwdsd2 foldId 1
     * ns:ads5wwdsd2 phaseId 1
     * ns:ads5wwdsd2 instances 12542
     * ns:ads5wwdsd2 learnedConcept <>
     * <p/>
     * Let's see... what information do i want?
     * nkepr:phaseX a nkebo:FoldPhase .
     * nkebr:phaseX hasFoldId 1^^xsd:integer .
     * nkebr:phaseX hasPhaseId 2^^xsd:integer .
     *
     * @param con
     * @param basePath
     * @param numFolds
     * @param numPhases
     * @param category
     * @param pool
     * @param random
     * @param experimentName
     * @param phaseReconfigurator
     * @throws Exception
     */

    public static void runBenchmark(BenchmarkContext con, String basePath, String namespace, int numFolds, int numPhases, Resource category, Sample<Resource> pool, Random random, String experimentName, IPhaseReconfigurator phaseReconfigurator) //ApplicationContext context)
            throws Exception
    {
        String experimentPath = basePath + "/" + experimentName;


        logger.info("Entering category " + category);


        String simpleCategoryName = category.toString().replace("http://dbpedia.org/resource/Category:", "");
        String encodedSimpleCategoryName = URLEncoder.encode(simpleCategoryName, "UTF-8");

        String categoryPath = experimentPath + "/" + encodedSimpleCategoryName;
        namespace = namespace + encodedSimpleCategoryName + "/";

        // Remove if exists, since we will be appending all our folds to it
        File file = new File(categoryPath + "-stats.nt");

        if (file.exists()) {
            file.delete();
        }


        Sample<Resource> all = Sample.create(
                MySparqlTasks.getDBpediaCategoryMembers(con.getSparqlEndpoint(), con.getGraphNames(), category),
                null);

        SerializationUtils.serializeXml(Sample.createCopy(all), new File(categoryPath + "/all.xml"), true);

        Sample<Resource> allTyped = Sample.create(
                MySparqlTasks.getTypedDBpediaCategoryMembers(con.getSparqlEndpoint(), con.getGraphNames(), category),
                null);

        SerializationUtils.serializeXml(Sample.createCopy(all), new File(categoryPath + "/allTyped.xml"), true);

        logger.info("Typed Members: " + ClausBenchmark.toMyString(allTyped.getPositives()));


        Sample<Resource> untyped = Sample.create(
                Sets.difference(all.getPositives(), allTyped.getPositives()),
                null
        );

        //pool.getPositives().addAll(allTyped.getPositives());
        //Sample<Resource> pool =  PoolUtils.generateNavigationPool(con.getSparqlEndpoint(), con.getGraphNames(), category, allTyped.getPositives(), random);
        logger.info("poolPositives: " + ClausBenchmark.toMyString(pool.getPositives()));
        logger.info("poolNegatives: " + ClausBenchmark.toMyString(pool.getNegatives()), 512);

        SerializationUtils.serializeXml(Sample.createCopy(pool), new File(categoryPath + "/navigationPool.xml"), true);


        ExperimentStatisticsCollector collector = new ExperimentStatisticsCollector(experimentName, numPhases);

        KFoldCollection<Resource> kFolds;
        try {
            kFolds = KFoldCollection.create(numFolds, pool, 0.5f, random);
        } catch (Throwable e) {
            logger.error("Error creating folds", e);
            return;
        }

        int foldId = 0;
        for (KFoldContext<Resource> kFold : kFolds) {
            try {

                ++foldId;

                logger.info("  Entering fold " + foldId);
                logger.info("    testPositives:  " + ClausBenchmark.toMyString(kFold.getTestSample().getPositives()));
                logger.info("    testNegatives:  " + ClausBenchmark.toMyString(kFold.getTestSample().getNegatives()));
                logger.info("    foldPositives:  " + ClausBenchmark.toMyString(kFold.getTrainSample().getPositives()));
                logger.info("    foldNegatives:  " + ClausBenchmark.toMyString(kFold.getTrainSample().getNegatives()));

                String foldPath = categoryPath + foldId;

                Sample<Resource> init = new Sample<Resource>(
                        RandomUtils.randomSampleSet(kFold.getTrainSample().getPositives(), 5, random),
                        RandomUtils.randomSampleSet(kFold.getTrainSample().getNegatives(), 5, random));

                //PhaseIterator phaseIterator = new PhaseIterator(allTyped, kFold.getTrainSample(), init, random, con.getLearner(), con.getClassifier());
                PhaseSet<Resource, EvaluatedDescription> phaseSet = new PhaseSet<Resource, EvaluatedDescription>(allTyped, kFold.getTrainSample(), init, con.getLearner(), con.getClassifier(), new RandomNPickingStrategy(5, random));


                for (int i = 0; i < numPhases; ++i) {
                    logger.info("    Entering phase " + (i + 1));

                    if (phaseReconfigurator != null) {
                        phaseReconfigurator.reconfigure(i);
                    }

                    phaseSet.next();

                    int phaseId = i + 1; //phaseSet.getPhaseId();
                    Model model = ModelFactory.createDefaultModel();
                    Resource phaseRes = ResourceFactory.createResource(namespace + "fold/" + foldId + "/phase/" + phaseId);

                    //logger.info("PHASE RES ="  + phaseRes);

                    try {

                        //String phasePath = foldPath + "-" + i + "-";

                        logger.info("      trainPositives: " + ClausBenchmark.toMyString(phaseSet.getCurrent().getPositives()));
                        logger.info("      trainNegatives: " + ClausBenchmark.toMyString(phaseSet.getCurrent().getNegatives()));


                        phaseSet.learn();
                        String readableConcept = phaseSet.getLearnedDescription().getDescription().toKBSyntaxString();
                        logger.info("      learned:        " + readableConcept);

                        phaseSet.classify();

                        Set<Resource> examples = phaseSet.getExamples();
                        int numRetrievedInstances = examples.size();
                        logger.info("      #instRetrieved: " + examples.size());


                        // Counting all instances may result in a time out of the query.
                        int numInstances = -1;
                        try {
                            numInstances = con.getClassifier().countInstances(phaseSet.getLearnedDescription());
                            logger.info("      #instActual:    " + numInstances);
                        } catch (Throwable t) {
                            logger.warn("Could not retrieve actual count of resources", t);
                        }


                        // Create a set of examples that does not contain the untyped positives, as those positives
                        // aren't actually wrong
                        Set<Resource> typedExamples = Sets.difference(examples, untyped.getPositives());

                        //Sample<Resource> allTypedX = Sample.create(allTyped.getPositives(), Sets.difference(typedExamples, allTyped.getPositives()));

                        SampleStats<Resource> allStats = SampleStats.create(all, examples);
                        SampleStats<Resource> allTypedStats = SampleStats.create(allTyped, typedExamples);

                        SampleStats<Resource> testAllStats = SampleStats.create(kFold.getTestSample(), examples, all);
                        SampleStats<Resource> testAllTypedStats = SampleStats.create(kFold.getTestSample(), typedExamples, allTyped);


                        BenchmarkUtils.add(collector, i, "all", allStats);
                        BenchmarkUtils.add(collector, i, "allTyped", allTypedStats);
                        BenchmarkUtils.add(collector, i, "testAll", testAllStats);
                        BenchmarkUtils.add(collector, i, "typedAllTyped", testAllTypedStats);

                        double learningAccuracy = phaseSet.getLearnedDescription().getAccuracy();
                        logger.info("      accuracy:       " + learningAccuracy);
                        logger.info("      all:            " + allStats);
                        logger.info("      allTyped:       " + allTypedStats);
                        logger.info("      testAll:        " + testAllStats);
                        logger.info("      testAllTyped:   " + testAllTypedStats);


                        // RDF output
                        model.add(phaseRes, RDF.type, Vocab.Phase);
                        model.add(phaseRes, Vocab.hasCategoryId, category);
                        model.add(phaseRes, Vocab.hasFoldId, ResourceFactory.createTypedLiteral(foldId));
                        model.add(phaseRes, Vocab.hasPhaseId, ResourceFactory.createTypedLiteral(phaseId));

                        model.add(phaseRes, Vocab.hasTrainPositives, ResourceFactory.createTypedLiteral(ClausBenchmark.toMyString(phaseSet.getCurrent().getPositives())));
                        model.add(phaseRes, Vocab.hasTrainNegatives, ResourceFactory.createTypedLiteral(ClausBenchmark.toMyString(phaseSet.getCurrent().getNegatives())));

                        writeValue(model, phaseRes, Vocab.LearningAccuracy, ResourceFactory.createTypedLiteral(learningAccuracy));
                        writeValue(model, phaseRes, Vocab.InstanceCount, ResourceFactory.createTypedLiteral(numInstances));
                        writeValue(model, phaseRes, Vocab.RetrievedInstanceCount, ResourceFactory.createTypedLiteral(numRetrievedInstances));
                        writeValue(model, phaseRes, Vocab.LearnedConcept, ResourceFactory.createPlainLiteral(readableConcept));
                        writeValue(model, phaseRes, Vocab.NumTrainPositive, ResourceFactory.createTypedLiteral(phaseSet.getCurrent().getPositives().size()));
                        writeValue(model, phaseRes, Vocab.NumTrainNegative, ResourceFactory.createTypedLiteral(phaseSet.getCurrent().getNegatives().size()));
                        //write(model, phaseRes, Vocab.StatsAll, allStats);
                        write(model, phaseRes, Vocab.StatsAllTyped, allTypedStats);
                        //write(model, phaseRes, Vocab.StatsTest, testAllStats);
                        write(model, phaseRes, Vocab.StatsTestTyped, testAllTypedStats);

                        //model.write(System.out, "N-TRIPLE");
                        String label = "BAD";
                        if (allTypedStats.getFMeasure() >= 0.5 && phaseSet.getLearnedDescription().getAccuracy() > 0.90) {
                            label = "GOOD";
                        }

                        logger.info("      [" + label + "] " + category + " --- " + allTypedStats);
                    }
                    catch (Throwable t) {
                        model.add(phaseRes, Vocab.hasError, ExceptionUtils.getFullStackTrace(t));
                        throw new Exception("Error in phase " + phaseId, t);
                    }
                    finally {
                        //File file = new File(categoryPath + "-stats.nt");
                        file.getParentFile().mkdirs();
                        OutputStream out = new FileOutputStream(file, true);
                        model.write(out, "N-TRIPLE");
                        out.flush();
                        out.close();
                        model.removeAll();
                    }
                    //SerializationUtils.serializeXml(phaseResult, new File(phasePath + "phaseResult.xml"), true);
                }
            } catch (Throwable t) {
                logger.warn("Error in fold " + foldId + ". Moving to next one (if there is).", t);
            }


            BenchmarkUtils.writeOutStatistics(categoryPath + "table", collector.getTable());
        }
    }
}
